{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ab6e9601-67d4-49a2-830d-9710fdf1d039",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_dataset\n",
    "from evaluate import evaluator, combine\n",
    "from transformers import pipeline\n",
    "from transformers.utils import logging\n",
    "logging.set_verbosity_error() \n",
    "\n",
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "660d09a5-770e-4096-9395-b4557045bd1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "sentences = load_dataset(\"rotten_tomatoes\", split=\"test\").select(range(5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1185d65f-09ec-49a2-a842-e830eab48ac4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "lovingly photographed in the manner of a golden book sprung to life , stuart little 2 manages sweetness largely without stickiness .\n",
      "consistently clever and suspenseful .\n",
      "it's like a \" big chill \" reunion of the baader-meinhof gang , only these guys are more harmless pranksters than political activists .\n",
      "the story gives ample opportunity for large-scale action and suspense , which director shekhar kapur supplies with tremendous skill .\n",
      "red dragon \" never cuts corners .\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[None, None, None, None, None]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[print(sentence) for sentence in sentences['text']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7cb8499c-4ae3-4f79-a935-68359c30485b",
   "metadata": {},
   "outputs": [],
   "source": [
    "roberta_pipe = pipeline(\"sentiment-analysis\", model=\"textattack/roberta-base-rotten-tomatoes\", device=device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "7d87173b-c3b5-422c-b4bb-180301fea60e",
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = roberta_pipe(sentences['text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e44c5698-ccd1-4fbd-b41d-2da2e110b127",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "actual:1 predicted:1 \t lovingly photographed in the manner of a golden book sprung to life , stuart little 2 manages sweetness largely without stickiness . \n",
      "actual:1 predicted:1 \t consistently clever and suspenseful . \n",
      "actual:1 predicted:0 \t it's like a \" big chill \" reunion of the baader-meinhof gang , only these guys are more harmless pranksters than political activists . \n",
      "actual:1 predicted:1 \t the story gives ample opportunity for large-scale action and suspense , which director shekhar kapur supplies with tremendous skill . \n",
      "actual:1 predicted:1 \t red dragon \" never cuts corners . \n"
     ]
    }
   ],
   "source": [
    "for idx, _sentence in enumerate(sentences['text']):\n",
    " print(f\"actual:{sentences['label'][idx]} \"\n",
    " f\"predicted:{'1' if predictions[idx]['label']=='LABEL_1' else '0'} \\t {_sentence} \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "33c3e54d-4c7e-4214-9e1c-1b20609571c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences = load_dataset(\"rotten_tomatoes\", split=\"test\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "64fe1127-b6e6-4066-b75e-8da4f4dc0b37",
   "metadata": {},
   "outputs": [],
   "source": [
    "task_evaluator = evaluator(\"sentiment-analysis\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "5a06940e-ec86-4129-a323-8635fb48ed46",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0a3a6eb9fb9c426aa6304483258b046c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading builder script:   0%|          | 0.00/7.55k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "30aa58c74a29485f905bf8a819640b7e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading builder script:   0%|          | 0.00/7.36k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "28a20ad17b314d059920478a95732452",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading builder script:   0%|          | 0.00/6.77k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "eval_results = task_evaluator.compute(\n",
    "   model_or_pipeline=roberta_pipe,\n",
    "   data=sentences,\n",
    "   metric=combine([\"accuracy\", \"precision\", \"recall\", \"f1\"]),\n",
    "   label_mapping={\"LABEL_0\": 0, \"LABEL_1\": 1})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "726a6b11-550f-4d33-89f7-dd25463a8df1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'accuracy': 0.8874296435272045, 'precision': 0.9222903885480572, 'recall': 0.8461538461538461, 'f1': 0.8825831702544031, 'total_time_in_seconds': 25.49598717000117, 'samples_per_second': 41.81050111502511, 'latency_in_seconds': 0.023917436369607106}\n"
     ]
    }
   ],
   "source": [
    "print(eval_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eb9caddc-8f1e-43ab-b95f-984dc2331287",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
